const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

async function main() {
  const fileName = `coolshell.json`;
  let totalBlogs = [];

  const browser = await puppeteer.launch({headless: true, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});
  page.on('console', msg => {
    console.log(msg.text());
  });

  page.on('pageerror', msg => {
    console.log(msg);
  });

  let url = 'https://coolshell.cn/featured';
  await page.goto(url, {waitUntil: 'domcontentloaded'});
  await page.waitFor(1000);

  totalBlogs = await page.$$eval('#post-4194 > div > ul > li > a', async (eles) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.innerText.trim();
      let content = ele.href;
      let source = page.querySelector('#author').innerText.trim();
      tmpBlogs.push({
        title: title,
        content: content,
        tab: 'coolshell',
        sub_tab: 'coolshell',
        source: source
      });
    });
    return tmpBlogs;
  });

  let writerStream = fs.createWriteStream(`crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');

  console.log(errLogs);

  // browser.close();
};


main();


